package com.fwmagic.spark.structuredstreaming

import org.apache.spark.sql.streaming.StreamingQuery
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}

object StructuredStreamingWC {

  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName("StructuredStreamingWC")
      .master("local[*]")
      .getOrCreate()

    import spark.implicits._

    //创建一个DataFrame输入流来读取socket中的每一行数据
    val lines: DataFrame = spark.readStream
      .format("socket")
      .option("host", "localhost")
      .option("port", "9999")
      .load()

    //空格切割行，获取单词数据集
    val words: Dataset[String] = lines.as[String].flatMap(_.split(" "))

    //生产运行次数
    val wordCounts: DataFrame = words.groupBy("value").count()

    //将运行计算结果打印到控制台
    val query: StreamingQuery = wordCounts.writeStream
      .outputMode("complete")
      .format("console")
      .start()

    query.awaitTermination()

  }

}
